{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bert\n",
    "import bert.modeling, bert.tokenization\n",
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/home/husein/parsing/self-attentive-parser/src\")\n",
    "sys.path.append(\"/home/husein/parsing/self-attentive-parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import itertools\n",
    "import os.path\n",
    "import time\n",
    "import shutil\n",
    "import re\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import torch.optim.lr_scheduler\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import evaluate\n",
    "import trees_newline as trees\n",
    "import vocabulary\n",
    "import nkutil\n",
    "import parse_nk_bert_base as parse_nk\n",
    "tokens = parse_nk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if parse_nk.use_cuda:\n",
    "    info = torch.load('models/en_bert_dev=80.35.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = parse_nk.NKChartParser.from_spec(info['spec'], info['state_dict'])\n",
    "bert_model = info['spec']['hparams']['bert_model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_treebank = trees.load_trees('test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "sd = parser.state_dict()\n",
    "\n",
    "LABEL_VOCAB = [x[0] for x in sorted(parser.label_vocab.indices.items(), key=lambda x: x[1])]\n",
    "TAG_VOCAB = [x[0] for x in sorted(parser.tag_vocab.indices.items(), key=lambda x: x[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert(input_ids, word_end_mask):\n",
    "    # We can derive input_mask from either input_ids or word_end_mask\n",
    "    input_mask = (1 - tf.cumprod(1 - word_end_mask, axis=-1, reverse=True))\n",
    "    token_type_ids = tf.zeros_like(input_ids)\n",
    "    bert_model = make_bert_instance(input_ids, input_mask, token_type_ids)\n",
    "\n",
    "    bert_features = bert_model.get_sequence_output()\n",
    "    bert_features_packed = tf.gather(\n",
    "        tf.reshape(bert_features, [-1, int(bert_features.shape[-1])]),\n",
    "        tf.to_int32(tf.where(tf.reshape(word_end_mask, (-1,))))[:,0])\n",
    "    projected_annotations = tf.matmul(\n",
    "        bert_features_packed,\n",
    "        tf.constant(sd['project_bert.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    # input_mask is over subwords, whereas valid_mask is over words\n",
    "    sentence_lengths = tf.reduce_sum(word_end_mask, -1)\n",
    "    valid_mask = (tf.range(tf.reduce_max(sentence_lengths))[None,:] < sentence_lengths[:, None])\n",
    "    dim_padded = tf.shape(valid_mask)[:2]\n",
    "    mask_flat = tf.reshape(valid_mask, (-1,))\n",
    "    dim_flat = tf.shape(mask_flat)[:1]\n",
    "    nonpad_ids = tf.to_int32(tf.where(mask_flat)[:,0])\n",
    "\n",
    "    return projected_annotations, nonpad_ids, dim_flat, dim_padded, valid_mask, sentence_lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "position_table = tf.constant(sd['embedding.position_table'].cpu().numpy(), name=\"position_table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_layer_norm(input, torch_name, name):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=tf.constant(sd[f'{torch_name}.b_2'].cpu().numpy(), name=f\"{name}/offset\"),\n",
    "        scale=tf.constant(sd[f'{torch_name}.a_2'].cpu().numpy(), name=f\"{name}/scale\"),\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "\n",
    "def make_heads(input, shape_bthf, shape_xtf, torch_name, name):\n",
    "    res = tf.matmul(input,\n",
    "        tf.constant(sd[torch_name].cpu().numpy().transpose((1,0,2)).reshape((512, -1)), name=f\"{name}/W\"))\n",
    "    res = tf.reshape(res, shape_bthf)\n",
    "    res = tf.transpose(res, (0,2,1,3)) # batch x num_heads x time x feat\n",
    "    res = tf.reshape(res, shape_xtf) # _ x time x feat\n",
    "    return res\n",
    "\n",
    "def make_attention(input, nonpad_ids, dim_flat, dim_padded, valid_mask, torch_name, name):\n",
    "    input_flat = tf.scatter_nd(indices=nonpad_ids[:, None], updates=input, shape=tf.concat([dim_flat, tf.shape(input)[1:]], axis=0))\n",
    "    input_flat_dat, input_flat_pos = tf.split(input_flat, 2, axis=-1)\n",
    "\n",
    "    shape_bthf = tf.concat([dim_padded, [8, -1]], axis=0)\n",
    "    shape_bhtf = tf.convert_to_tensor([dim_padded[0], 8, dim_padded[1], -1])\n",
    "    shape_xtf = tf.convert_to_tensor([dim_padded[0] * 8, dim_padded[1], -1])\n",
    "    shape_xf = tf.concat([dim_flat, [-1]], axis=0)\n",
    "\n",
    "    qs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_qs1', f'{name}/q_dat')\n",
    "    ks1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_ks1', f'{name}/k_dat')\n",
    "    vs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_vs1', f'{name}/v_dat')\n",
    "    qs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_qs2', f'{name}/q_pos')\n",
    "    ks2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_ks2', f'{name}/k_pos')\n",
    "    vs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_vs2', f'{name}/v_pos')\n",
    "\n",
    "    qs = tf.concat([qs1, qs2], axis=-1)\n",
    "    ks = tf.concat([ks1, ks2], axis=-1)\n",
    "    attn_logits = tf.matmul(qs, ks, transpose_b=True) / (1024 ** 0.5)\n",
    "\n",
    "    attn_mask = tf.reshape(tf.tile(valid_mask, [1,8*dim_padded[1]]), tf.shape(attn_logits))\n",
    "    # TODO(nikita): use tf.where and -float('inf') here?\n",
    "    attn_logits -= 1e10 * tf.to_float(~attn_mask)\n",
    "\n",
    "    attn = tf.nn.softmax(attn_logits)\n",
    "\n",
    "    attended_dat_raw = tf.matmul(attn, vs1)\n",
    "    attended_dat_flat = tf.reshape(tf.transpose(tf.reshape(attended_dat_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_dat = tf.gather(attended_dat_flat, nonpad_ids)\n",
    "    attended_pos_raw = tf.matmul(attn, vs2)\n",
    "    attended_pos_flat = tf.reshape(tf.transpose(tf.reshape(attended_pos_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_pos = tf.gather(attended_pos_flat, nonpad_ids)\n",
    "\n",
    "    out_dat = tf.matmul(attended_dat, tf.constant(sd[f'{torch_name}.proj1.weight'].cpu().numpy().transpose()))\n",
    "    out_pos = tf.matmul(attended_pos, tf.constant(sd[f'{torch_name}.proj2.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_dense_relu_dense(input, torch_name, torch_type, name):\n",
    "    # TODO: use name\n",
    "    mul1 = tf.matmul(input, tf.constant(sd[f'{torch_name}.w_1{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul1b = tf.nn.bias_add(mul1, tf.constant(sd[f'{torch_name}.w_1{torch_type}.bias'].cpu().numpy()))\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, tf.constant(sd[f'{torch_name}.w_2{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul2b = tf.nn.bias_add(mul2, tf.constant(sd[f'{torch_name}.w_2{torch_type}.bias'].cpu().numpy()))\n",
    "    return mul2b\n",
    "\n",
    "def make_ff(input, torch_name, name):\n",
    "    # TODO: use name\n",
    "    input_dat, input_pos = tf.split(input, 2, axis=-1)\n",
    "    out_dat = make_dense_relu_dense(input_dat, torch_name, 'c', name=\"TODO_dat\")\n",
    "    out_pos = make_dense_relu_dense(input_pos, torch_name, 'p', name=\"TODO_pos\")\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_stacks(input, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks):\n",
    "    res = input\n",
    "    for i in range(num_stacks):\n",
    "        res = make_attention(res, nonpad_ids, dim_flat, dim_padded, valid_mask, f'encoder.attn_{i}', name=f'attn_{i}')\n",
    "        res = make_ff(res, f'encoder.ff_{i}', name=f'ff_{i}')\n",
    "    return res\n",
    "\n",
    "def make_layer_norm_with_constants(input, constants):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=constants[0],\n",
    "        scale=constants[1],\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "def make_flabel_with_constants(input, constants):\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='flabel')\n",
    "    return mul2b\n",
    "\n",
    "def make_ftag(input):\n",
    "    constants = (\n",
    "        tf.constant(sd['f_tag.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_tag.1.b_2'].cpu().numpy(), name=\"tag/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_tag.1.a_2'].cpu().numpy(), name=\"tag/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_tag.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='ftag')\n",
    "    return mul2b\n",
    "\n",
    "def make_flabel_constants():\n",
    "    return (\n",
    "        tf.constant(sd['f_label.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_label.1.b_2'].cpu().numpy(), name=\"label/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_label.1.a_2'].cpu().numpy(), name=\"label/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_label.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "\n",
    "def make_network():\n",
    "    # batch x num_subwords\n",
    "    input_ids = tf.placeholder(shape=(None, None), dtype=tf.int32, name='input_ids')\n",
    "    word_end_mask = tf.placeholder(shape=(None, None), dtype=tf.int32, name='word_end_mask')\n",
    "    input_dat, nonpad_ids, dim_flat, dim_padded, valid_mask, lengths = make_bert(input_ids, word_end_mask)\n",
    "    input_pos_flat = tf.tile(position_table[:dim_padded[1]], [dim_padded[0], 1])\n",
    "    input_pos = tf.gather(input_pos_flat, nonpad_ids)\n",
    "\n",
    "    input_joint = tf.concat([input_dat, input_pos], -1)\n",
    "    input_joint = make_layer_norm(input_joint, 'embedding.layer_norm', 'embedding/layer_norm')\n",
    "\n",
    "    word_out = make_stacks(input_joint, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks=parser.spec['hparams']['num_layers'])\n",
    "    word_out = tf.concat([word_out[:, 0::2], word_out[:, 1::2]], -1)\n",
    "\n",
    "    # part-of-speech predictions\n",
    "    ftag = make_ftag(word_out)\n",
    "    tags_packed = tf.argmax(ftag, axis=-1)\n",
    "    tags = tf.reshape(\n",
    "        tf.scatter_nd(indices=nonpad_ids[:, None], updates=tags_packed, shape=dim_flat),\n",
    "        dim_padded\n",
    "        )\n",
    "    tags = tf.identity(tags, name=\"tags\")\n",
    "\n",
    "    fp_out = tf.concat([word_out[:-1,:512], word_out[1:,512:]], -1)\n",
    "\n",
    "    fp_start_idxs = tf.cumsum(lengths, exclusive=True)\n",
    "    fp_end_idxs = tf.cumsum(lengths) - 1 # the number of fenceposts is 1 less than the number of words\n",
    "\n",
    "    fp_end_idxs_uneven = fp_end_idxs - tf.convert_to_tensor([1, 0])\n",
    "\n",
    "    # Have to make these outside tf.map_fn for model compression to work\n",
    "    constants = make_flabel_constants()\n",
    "\n",
    "    def to_map(start_and_end):\n",
    "        start, end = start_and_end\n",
    "        fp = fp_out[start:end]\n",
    "        flabel = make_flabel_with_constants(tf.reshape(fp[None,:,:] - fp[:,None,:], (-1, 1024)), constants)\n",
    "        actual_chart_size = end-start\n",
    "        flabel = tf.reshape(flabel, [actual_chart_size, actual_chart_size, -1])\n",
    "        amount_to_pad = dim_padded[1] - actual_chart_size\n",
    "        # extra padding on the label dimension is for the not-a-constituent label,\n",
    "        # which always has a score of 0\n",
    "        flabel = tf.pad(flabel, [[0, amount_to_pad], [0, amount_to_pad], [1, 0]])\n",
    "        return flabel\n",
    "\n",
    "    charts = tf.map_fn(to_map, (fp_start_idxs, fp_end_idxs), dtype=(tf.float32))\n",
    "    charts = tf.identity(charts, name=\"charts\")\n",
    "\n",
    "    return input_ids, word_end_mask, charts, tags\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert_instance(input_ids, input_mask, token_type_ids):\n",
    "    # Transfer BERT config into tensorflow implementation\n",
    "    config = bert.modeling.BertConfig.from_dict(parser.bert.config.to_dict())\n",
    "    model = bert.modeling.BertModel(config=config, is_training=False,\n",
    "        input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)\n",
    "\n",
    "    # Next, transfer learned weights (after fine-tuning)\n",
    "    bert_variables = [v for v in tf.get_collection('variables') if 'bert' in v.name]\n",
    "    tf.variables_initializer(bert_variables).run()\n",
    "    \n",
    "    for variable in bert_variables:\n",
    "        name = variable.name.split(':')[0]\n",
    "        name = name.split('/')\n",
    "        array = variable.eval()\n",
    "        # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v\n",
    "        # which are not required for using pretrained model\n",
    "        if any(n in [\"adam_v\", \"adam_m\"] for n in name):\n",
    "            print(\"Skipping {}\".format(\"/\".join(name)))\n",
    "            continue\n",
    "        pytorch_var = parser\n",
    "        for m_name in name:\n",
    "            if re.fullmatch(r'[A-Za-z]+_\\d+', m_name):\n",
    "                l = re.split(r'_(\\d+)', m_name)\n",
    "            else:\n",
    "                l = [m_name]\n",
    "            if l[0] == 'kernel' or l[0] == 'gamma':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'output_bias' or l[0] == 'beta':\n",
    "                pytorch_var = getattr(pytorch_var, 'bias')\n",
    "            elif l[0] == 'output_weights':\n",
    "                pytorch_var = getattr(pytorch_var, 'weight')\n",
    "            elif l[0] == 'cls':\n",
    "                pytorch_var = getattr(pytorch_var, 'cls')\n",
    "            else:\n",
    "                pytorch_var = getattr(pytorch_var, l[0])\n",
    "            if len(l) >= 2:\n",
    "                try:\n",
    "                    num = int(l[1])\n",
    "                    pytorch_var = pytorch_var[num]\n",
    "                except Exception as e:\n",
    "                    print(pytorch_var)\n",
    "                    raise\n",
    "        if m_name[-11:] == '_embeddings':\n",
    "            pytorch_var = getattr(pytorch_var, 'weight')\n",
    "        elif m_name == 'kernel':\n",
    "            pytorch_var = pytorch_var.t()\n",
    "        try:\n",
    "            assert pytorch_var.shape == array.shape\n",
    "        except AssertionError as e:\n",
    "            print(e)\n",
    "            e.args += (pytorch_var.shape, array.shape)\n",
    "            raise\n",
    "            \n",
    "        variable.load(pytorch_var.detach().cpu().numpy())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:233: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-12-f98425b040db>:54: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n",
      "WARNING:tensorflow:From <ipython-input-9-209ca551d3eb>:10: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-9-209ca551d3eb>:10: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "WARNING:tensorflow:From <ipython-input-11-dc457fb7a117>:46: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "the_inp_tokens, the_inp_mask, the_out_chart, the_out_tags = make_network()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from parse_nk import BERT_TOKEN_MAPPING\n",
    "def bertify_batch(sentences):\n",
    "    all_input_ids = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_word_end_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "\n",
    "    subword_max_len = 0\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "        tokens = []\n",
    "        word_end_mask = []\n",
    "\n",
    "        tokens.append(\"[CLS]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        cleaned_words = []\n",
    "        for word in sentence:\n",
    "            word = BERT_TOKEN_MAPPING.get(word, word)\n",
    "            if word == \"n't\" and cleaned_words:\n",
    "                cleaned_words[-1] = cleaned_words[-1] + \"n\"\n",
    "                word = \"'t\"\n",
    "            cleaned_words.append(word)\n",
    "\n",
    "        for word in cleaned_words:\n",
    "            word_tokens = parser.bert_tokenizer.tokenize(word)\n",
    "            for _ in range(len(word_tokens)):\n",
    "                word_end_mask.append(0)\n",
    "            word_end_mask[-1] = 1\n",
    "            tokens.extend(word_tokens)\n",
    "        tokens.append(\"[SEP]\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        input_ids = parser.bert_tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "        subword_max_len = max(subword_max_len, len(input_ids))\n",
    "\n",
    "        all_input_ids[snum, :len(input_ids)] = input_ids\n",
    "        all_word_end_mask[snum, :len(word_end_mask)] = word_end_mask\n",
    "\n",
    "    all_input_ids = all_input_ids[:, :subword_max_len]\n",
    "    all_word_end_mask = all_word_end_mask[:, :subword_max_len]\n",
    "    return all_input_ids, all_word_end_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28/28 [00:02<00:00, 10.66it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "eval_batch_size = 16\n",
    "test_predicted = []\n",
    "for start_index in tqdm(range(0, len(test_treebank), eval_batch_size)):\n",
    "    subbatch_trees = test_treebank[start_index:start_index+eval_batch_size]\n",
    "    subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "    inp_val_tokens, inp_val_mask = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "    out_val_chart, out_val_tags = sess.run((the_out_chart, the_out_tags), \n",
    "                                       {the_inp_tokens: inp_val_tokens, the_inp_mask: inp_val_mask})\n",
    "    trees = []\n",
    "    scores = []\n",
    "    for snum, sentence in enumerate(subbatch_sentences):\n",
    "        chart_size = len(sentence) + 1\n",
    "        tf_chart = out_val_chart[snum,:chart_size,:chart_size,:]\n",
    "        sentence = list(zip([TAG_VOCAB[idx] for idx in out_val_tags[snum,1:chart_size]], [x for x in sentence]))\n",
    "        tree, score = parser.decode_from_chart(sentence, tf_chart)\n",
    "        trees.append(tree)\n",
    "        scores.append(score)\n",
    "    test_predicted.extend([p.convert() for p in trees])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<trees_newline.InternalTreebankNode object at 0x7f49be275da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445e1048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49be29a470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445e1c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49be2328d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445e56a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0b7400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445e81d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0b8cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445e8d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0bd5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445ea6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0bee80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445ed0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0c2828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445edac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0c6358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445f0748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0c7f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445f3358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0ccb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445f3fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0d0780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445f7cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0d4320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445fa940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0d6f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445fc5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0dab70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445802e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0df5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944580ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0e4208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d94e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0e53c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944583be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0e90b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445ce668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49be2cc668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d6240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd8f8160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d6780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0ec080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d8390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0ecf60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d8ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0ede80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445dc320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0efda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445dc828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0f1c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445dcf28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0f2ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494458c668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0f4ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494458ccf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd9759e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494458f400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd970eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494458fac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd9769e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445914e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd96f4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445919e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd96ff98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445db898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd973ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445dbda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd9715c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445892b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd965128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944589780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd965ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944589c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd9646a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445971d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd956748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445976d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd958780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494459a588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bd94f898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494459c2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc076a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494459f0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc078c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494459f9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc07ae10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445a3278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc07e0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445a3ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc080278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445a54a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc082438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445a5da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc083630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445a86a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc085898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445abac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc089828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445b0278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc08f438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445b0f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc091400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445b5208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc094400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445b5860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc096400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445b5e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc097438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445b94e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0993c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445b9b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc09b390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445bc160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc09d3c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445bc780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc09f400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445bcd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0a1ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494453f438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0a4a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944541160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0a7198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944541d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0aa160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944544128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0aa6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944547080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0aac18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d05c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0ad160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d0cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0ad668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d0f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0adba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d30f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0b0198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d36a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0b0710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d3160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0b0c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d37f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc036240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445d39e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc03a358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445ab3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc03ada0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494463c9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc03b860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944547470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc03d2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445479b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc03dd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944547e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc03f7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944548390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc041240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944548860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc041c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944548d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc043710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494455d160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc044cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494455d5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc047550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494455df60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc047da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944560390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0486a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445606d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc048f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944560a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc04a7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944560e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc04d0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944564278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc04d9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944564630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc04e2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944564a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc050a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944564e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc057080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445669b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc05b748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944569be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc05ecf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494456dd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0663c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944570f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc06aa90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944576160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc071128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494450ca58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49bc0737b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494450eba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07bddd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445125f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07c2400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944515048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07c3b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944515a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07c9198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944517550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07ca8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944517f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07ccf28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494451ab70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07d05f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494451c5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07d2d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445210f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07da518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944521c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07ddc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944528160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07e54a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494452c5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07e8ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494452fa20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07ef3c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944533eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07f4ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944538320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07f69e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944551128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07f87f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944551978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07fa630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445550f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b077d470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445557f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b077f2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944555fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07810f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444fe240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0781ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444fecf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0783d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944503390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0788518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944503a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0789c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445054e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b078f438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445088d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0790b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445084a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0795358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444bd278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0796b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444bdcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b079c320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444c0710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b079db00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444c3160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a0160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944579c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a0780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944579c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a0da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944579fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a33c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445577f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a39b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944557b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a3f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445572e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a45f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445570f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a4c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944557438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07a8c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445575f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07ac748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445c3d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07afd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444c8668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07b8198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444c8fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07b9ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444cc780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b073d400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d0438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b073d908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d0630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b073de10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d0828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b073e320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d72b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b073e7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d74a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b073ecf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d76a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0740208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d7898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0740710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d7a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0747048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444d7c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0747e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444db0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b074acf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444db7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b074cac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444dbf60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b074e8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444de6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b074f748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444dedd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0752588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444e1518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0754390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444e1c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07571d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444e43c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07599b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444e4ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b075d278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444e7470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b075fb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494447f320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07635f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494447fdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0765eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944481898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0769860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444852b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b076e278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944485cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b076fb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944489710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0774e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494448b1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0778710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494448e240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b077af28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494448ee10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06ff7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444929e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07030b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944493630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0705908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944496240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b070a1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944496e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b070ca20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494449aa90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07102e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494449c6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07155c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494449f320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0718828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444a7470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b071ed30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444a9668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07242b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444ab860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b07297b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444ae9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b072cda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444b1ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0733390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444b3dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0737978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444b6f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0738f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444bc198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06be5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444bcb38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06bfba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944440438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06c4160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944440cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06c67b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944442780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06c7da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944446128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06cc390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944446b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06cd9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494444a550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06d3080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494444aeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06d3e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494445de10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06d4c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494445f400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06d6a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494445f9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06d87f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494445ff60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06db630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944462588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06dd470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944462b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06df2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944465198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06e49b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944465780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06ee128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494446c080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06f37f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494446e940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06f8f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444751d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0682748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944478a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0687dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443ff390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0690550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444f9c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0695c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444075c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b069c8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944409e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06a34e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494450c9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06aa0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444eb080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06adc50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944410358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06b4860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444557f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06ba438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444509e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b063efd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944414da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0644c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944419208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0648748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494441b6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06492b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494441bb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0649dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494441bf28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b064b940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494441f3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b064e4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494441f860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0650048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494441fcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0650b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944422128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06516d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944422550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0656358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444229b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0658f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494443cba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b065cbe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443c0908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b065f898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443c3630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0665550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443c5400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0669128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443c8160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b066be10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443c8eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b066fa58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443ccc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0674710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443ce978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0678630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443d26d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b067b518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443d5470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0602dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443d8208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b060c5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443dbb70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0610e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443e2518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b061b5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443e5f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0621dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443eb898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b062a5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443f1208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0630da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944430860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b06395c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944435358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b063bdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443f4dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05c0748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944437978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05c70f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494442bc88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05cba58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494437dfd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05d22e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443823c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05d6be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944386710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05dc588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944388a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05df908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494438cdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05e1c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944390780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05e2fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944392128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05e7400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443929e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05e95c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944396320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05ea7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944396ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05ed908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443983c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05efa90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443ab208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05f0c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443aba90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05f4e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443af320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05fbf28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443afba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05834e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443b4828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b058aac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443ba0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0593048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494439ea20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b059a588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944341400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b059fc18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944344d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05a82e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445c9748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05ac940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944350160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05b2da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443522b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05b6dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944355ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05bb668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443589e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b053df28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494435b588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05417f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494435e198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05460f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444f8a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0548940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444fb390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b054c1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944424a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b054da90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494442c550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0553358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494442cda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0554828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444a72e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0556dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443a4898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b055a2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443a4dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b055d7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443a8e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b055fcc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494439c358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05631d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494439ccf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0565668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944360748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0567ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944364160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b056bb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944364b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b056fb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443669e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0572b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944369898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0578b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944372c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b057cac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944375978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0500a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494437a630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05049b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494437c320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05099b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944300048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b050d828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944300eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b050fdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944303c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b05150b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443056d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0515e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944305f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b051a780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944308c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b051e470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494430c668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0523198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494430f278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0524e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494430fe48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0528b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944312a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b052d828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944316630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0530550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443191d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0534d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444f65f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0537588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494436cf28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0539dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944372a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04be5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494431f710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04c0da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944321278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04c55c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944321e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04c6e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944323978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04cb668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49443264e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04cdba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944329048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04d2128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944329a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04d36d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494432b4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04d7cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494432be48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04db208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494432f860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04dd780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944332320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04dfda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944332d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04e2400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944334710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04e76a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442bda20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04eb940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442bf9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04eebe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442c3908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04f3e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442c5908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04fb080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442c88d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04fc1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442cb940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0480f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442cf0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04844e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442d25c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04879e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442d2f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0489e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442d6898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b048d208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442d8208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b048e6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442d8b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0491ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442dc470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0495160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442dce10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04976d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442de780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b049b828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442e20f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b049f9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944338358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04a3b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442e5978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04a6c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442e8780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04abd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442ec5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04b0e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442ef550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04b3f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442f14a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04ba160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442f54a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b043d518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442f8438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b043e8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442f8d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0440c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442fa630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0445048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442faef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0446400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494427e898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0448780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442811d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b044bb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944281ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b044ce80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442833c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04518d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944283c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04549e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442988d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0459c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494429a908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b045ceb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494429e7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0465198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442a1860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04674a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442a37b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b046c630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442a77f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04708d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442aa860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0474be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442ad860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b047b2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442b0898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03fde48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442b3908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04044e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442b6c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0408ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442b9d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b040f240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442bcf60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0413940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442430f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0417f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944245320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b041e6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442484e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0420b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444eaef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0421f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444f6978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0427438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944286ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0428860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494428c358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b042cc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494428cbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b042f128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494428e518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b04324e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494428ee80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0434940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944291780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0436da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944295cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b043b8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944295080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03c11d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494424def0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03c4a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944250ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03c9fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944254fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03cd1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944258fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03cfd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494425b748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03d3278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494425e2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03d3a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494454ee80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03d6278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494454e978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03d6a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445db668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03d7278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445db2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03d7a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445794e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03da208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49445798d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03da9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444ea0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03dc208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444ea3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03e1c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49444eaeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03e6860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944266e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03e8c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944268ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03ec828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494426a6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03f14a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494426d518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03f40b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494426f358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03f7cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944272198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03fb978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944272f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b037f550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944203b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03851d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944206940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0386e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442097b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0389f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494420c5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0391048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494420f550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0394198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442114a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b039a208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944215358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b039d240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944218240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03a2358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494421a240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03a54e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494421f198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03a8780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442220f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03aab70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944222a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03acfd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944225518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03b03c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944225eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03b3710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944228860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03b6b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494422c240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03b7e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442798d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b03bc240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944201588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0340e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944201c88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0344550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49441fd438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0346cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49441fda58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b034b4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494422f7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b034dc18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944233320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0352400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944233eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0353be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442359e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0358400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49442384e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b035abe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944238fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b035c7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494423cb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b035e358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49441bf0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b035eeb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49441bf5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0360a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49441bfba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0361630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f49441c20f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0364208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f494454ec50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0364d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944338c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f49b0365908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f4944338240>\n"
     ]
    }
   ],
   "source": [
    "test_fscore = evaluate.evalb('EVALB/', test_treebank[:len(test_predicted)], test_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(Recall=78.96, Precision=81.78, FScore=80.35, CompleteMatch=10.37, TaggingAccuracy=91.59)'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_names = [the_inp_tokens.name.split(':')[0], the_inp_mask.name.split(':')[0]]\n",
    "output_node_names = [the_out_chart.name.split(':')[0], the_out_tags.name.split(':')[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-19-f034bd79213f>:1: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 197 variables.\n",
      "INFO:tensorflow:Converted 197 variables to const ops.\n"
     ]
    }
   ],
   "source": [
    "graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, output_node_names)\n",
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "graph_def = TransformGraph(graph_def, input_node_names, output_node_names, [\n",
    "'strip_unused_nodes()',\n",
    "'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "'fold_constants()',\n",
    "'fold_old_batch_norms',\n",
    "'fold_batch_norms',\n",
    "'round_weights(num_steps=128)',\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('export/model.pb', 'wb') as f:\n",
    "    f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab.json', 'w') as fopen:\n",
    "    json.dump({'label': LABEL_VOCAB, 'tag': TAG_VOCAB}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<trees_newline.InternalTreebankNode at 0x7f49445e1048>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_predicted[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
